{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "39616240-0e59-45d8-88c1-35b87e340324",
   "metadata": {},
   "source": [
    "# Alpaca Lora Training"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20ff1154-f001-4310-af1e-97feb6a1f298",
   "metadata": {},
   "source": [
    "详细思路见：[https://github.com/unit-mesh/unit-minions](https://github.com/unit-mesh/unit-minions)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bedc0878-0e40-4632-8162-7d2227809657",
   "metadata": {},
   "source": [
    "## Setup LFS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "326a6f80-0723-40ee-812f-8164e6b038a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "!apt update\n",
    "!apt install git-lfs\n",
    "# check install\n",
    "!git lfs install"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5cdbf030",
   "metadata": {},
   "source": [
    "## Clone to Local"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "434bee3a-5913-48f4-86e3-16a3e42a2a16",
   "metadata": {},
   "outputs": [],
   "source": [
    "!git clone https://huggingface.co/decapoda-research/llama-7b-hf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d278f1e6",
   "metadata": {},
   "source": [
    "# Traing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "305b18a5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!git clone https://github.com/tloen/alpaca-lora"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d7b5f942-504e-492f-a25b-c6795a538ab2",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd alpaca-lora && pip install -r requirements.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5cce5ab-77d9-40b0-9fa4-61f3de1381f2",
   "metadata": {},
   "source": [
    "## Training in Local"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b7a120f-acef-4e77-822c-d298f4920304",
   "metadata": {},
   "outputs": [],
   "source": [
    "### Normal Traing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "712b1e50-fbb6-4ff1-bf09-62466be77e60",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 如果数据集太小，设置一下 val_set_size，比如：--val_set_size 500 \\\n",
    "!cd alpaca-lora && python finetune.py \\\n",
    "    --base_model '../llama-7b-hf' \\\n",
    "    --data_path '../userstory_detail.jsonl' \\\n",
    "    --output_dir './lora-alpaca'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1b42535-5b7c-44d4-bc8b-46ce02b4c7b9",
   "metadata": {},
   "source": [
    "### Training with Custom parapmters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "857ee84b",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "!cd alpaca-lora && python finetune.py \\\n",
    "    --base_model '../llama-7b-hf' \\\n",
    "    --data_path 'test_lora.jsonl' \\\n",
    "    --output_dir './lora-alpaca' \\\n",
    "    --batch_size 128 \\\n",
    "    --micro_batch_size 4 \\\n",
    "    --num_epochs 3 \\\n",
    "    --learning_rate 1e-4 \\\n",
    "    --cutoff_len 512 \\\n",
    "    --val_set_size 2000 \\\n",
    "    --lora_r 8 \\\n",
    "    --lora_alpha 16 \\\n",
    "    --lora_dropout 0.05 \\\n",
    "    --lora_target_modules '[q_proj,v_proj]' \\\n",
    "    --train_on_inputs \\\n",
    "    --group_by_length"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "58975bf9-4e94-4ace-b97c-47dada25a931",
   "metadata": {},
   "source": [
    "# 评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d66bef68-2894-4e2e-ae57-78e5ba485e46",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from peft import PeftModel\n",
    "import transformers\n",
    "import gradio as gr\n",
    "\n",
    "assert (\n",
    "    \"LlamaTokenizer\" in transformers._import_structure[\"models.llama\"]\n",
    "), \"LLaMA is now in HuggingFace's main branch.\\nPlease reinstall it: pip uninstall transformers && pip install git+https://github.com/huggingface/transformers.git\"\n",
    "from transformers import LlamaTokenizer, LlamaForCausalLM, GenerationConfig\n",
    "\n",
    "tokenizer = LlamaTokenizer.from_pretrained(\"./llama-7b-hf\")\n",
    "\n",
    "BASE_MODEL = \"./llama-7b-hf\"\n",
    "LORA_WEIGHTS = \"./alpaca-lora/lora-alpaca\"\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    device = \"cuda\"\n",
    "else:\n",
    "    device = \"cpu\"\n",
    "\n",
    "try:\n",
    "    if torch.backends.mps.is_available():\n",
    "        device = \"mps\"\n",
    "except:\n",
    "    pass\n",
    "\n",
    "if device == \"cuda\":\n",
    "    model = LlamaForCausalLM.from_pretrained(\n",
    "        BASE_MODEL,\n",
    "        load_in_8bit=False,\n",
    "        torch_dtype=torch.float16,\n",
    "        device_map=\"auto\",\n",
    "    )\n",
    "    model = PeftModel.from_pretrained(\n",
    "        model, LORA_WEIGHTS, torch_dtype=torch.float16, force_download=True\n",
    "    )\n",
    "elif device == \"mps\":\n",
    "    model = LlamaForCausalLM.from_pretrained(\n",
    "        BASE_MODEL,\n",
    "        device_map={\"\": device},\n",
    "        torch_dtype=torch.float16,\n",
    "    )\n",
    "    model = PeftModel.from_pretrained(\n",
    "        model,\n",
    "        LORA_WEIGHTS,\n",
    "        device_map={\"\": device},\n",
    "        torch_dtype=torch.float16,\n",
    "    )\n",
    "else:\n",
    "    model = LlamaForCausalLM.from_pretrained(\n",
    "        BASE_MODEL, device_map={\"\": device}, low_cpu_mem_usage=True\n",
    "    )\n",
    "    model = PeftModel.from_pretrained(\n",
    "        model,\n",
    "        LORA_WEIGHTS,\n",
    "        device_map={\"\": device},\n",
    "    )\n",
    "\n",
    "\n",
    "def generate_prompt(instruction, input=None):\n",
    "    if input:\n",
    "        return f\"\"\"Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n",
    "### Instruction:\n",
    "{instruction}\n",
    "### Input:\n",
    "{input}\n",
    "### Response:\"\"\"\n",
    "    else:\n",
    "        return f\"\"\"Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
    "### Instruction:\n",
    "{instruction}\n",
    "### Response:\"\"\"\n",
    "\n",
    "if device != \"cpu\":\n",
    "    model.half()\n",
    "model.eval()\n",
    "if torch.__version__ >= \"2\":\n",
    "    model = torch.compile(model)\n",
    "\n",
    "\n",
    "def evaluate(\n",
    "    instruction,\n",
    "    input=None,\n",
    "    temperature=0.1,\n",
    "    top_p=0.75,\n",
    "    top_k=40,\n",
    "    num_beams=4,\n",
    "    max_new_tokens=128,\n",
    "    **kwargs,\n",
    "):\n",
    "    prompt = generate_prompt(instruction, input)\n",
    "    inputs = tokenizer(prompt, return_tensors=\"pt\")\n",
    "    input_ids = inputs[\"input_ids\"].to(device)\n",
    "    generation_config = GenerationConfig(\n",
    "        temperature=temperature,\n",
    "        top_p=top_p,\n",
    "        top_k=top_k,\n",
    "        num_beams=num_beams,\n",
    "        **kwargs,\n",
    "    )\n",
    "    with torch.no_grad():\n",
    "        generation_output = model.generate(\n",
    "            input_ids=input_ids,\n",
    "            generation_config=generation_config,\n",
    "            return_dict_in_generate=True,\n",
    "            output_scores=True,\n",
    "            max_new_tokens=max_new_tokens,\n",
    "        )\n",
    "    s = generation_output.sequences[0]\n",
    "    output = tokenizer.decode(s)\n",
    "    return output.split(\"### Response:\")[1].strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6417ba24-7c62-46b6-9b15-6c6aedda2577",
   "metadata": {
    "execution": {
     "iopub.status.idle": "2023-04-03T14:00:48.986589Z",
     "shell.execute_reply": "2023-04-03T14:00:48.985592Z",
     "shell.execute_reply.started": "2023-04-03T14:00:27.211720Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I will participate in online forums and discussions.\n",
      "### Instruction:\n",
      "create userstory\n",
      "### Input:\n",
      " Animation and Comics:Participate in online forums and discussions\n"
     ]
    }
   ],
   "source": [
    "print(evaluate(\"create Agile user story for following topic: \", \" Animation and Comics:Participate in online forums and discussions\", 0.1, 0.75, 40, 4, 512))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2440efa9-eaea-4a2a-93bb-3ed71e7f8749",
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "# Run with Server"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5db00cfd",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "!git clone https://huggingface.co/spaces/tloen/alpaca-lora alpaca-lora-web"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b124ab96",
   "metadata": {},
   "source": [
    "## 配置源（按需）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b7f26d8",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "!pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20152e13",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cd alpaca-lora-web && pip install -r requirements.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "581e4fb5",
   "metadata": {},
   "source": [
    "## Run Server"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8cc8772b",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "# modify app.py huggingFace to Local\n",
    "# tokenizer = LlamaTokenizer.from_pretrained(\"../llama-7b-hf\")\n",
    "# BASE_MODEL = \"../llama-7b-hf\"\n",
    "!cd alpaca-lora-web && python app.py"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
